{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train and Evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 1/1 [00:08<00:00,  8.94s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished running strategy.\n",
      "Total episodes: 1 (1666 timesteps).\n",
      "Average reward: 1150.3350630537668.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>balance</th>\n",
       "      <th>net_worth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1661</th>\n",
       "      <td>9999.829222</td>\n",
       "      <td>10000.187790</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1662</th>\n",
       "      <td>9997.354264</td>\n",
       "      <td>9998.232905</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1663</th>\n",
       "      <td>9994.879919</td>\n",
       "      <td>9995.990028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1664</th>\n",
       "      <td>9991.169320</td>\n",
       "      <td>9992.191048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1665</th>\n",
       "      <td>9991.535304</td>\n",
       "      <td>9992.226192</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          balance     net_worth\n",
       "1661  9999.829222  10000.187790\n",
       "1662  9997.354264   9998.232905\n",
       "1663  9994.879919   9995.990028\n",
       "1664  9991.169320   9992.191048\n",
       "1665  9991.535304   9992.226192"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensorforce.agents import Agent\n",
    "from tensorforce.environments import Environment\n",
    "\n",
    "from tensortrade.environments import TradingEnvironment\n",
    "from tensortrade.exchanges.simulated import FBMExchange\n",
    "from tensortrade.features.scalers import MinMaxNormalizer\n",
    "from tensortrade.features.stationarity import FractionalDifference\n",
    "from tensortrade.features import FeaturePipeline\n",
    "from tensortrade.rewards import SimpleProfit\n",
    "from tensortrade.actions import DiscreteActions\n",
    "from tensortrade.strategies import TensorforceTradingStrategy\n",
    "\n",
    "normalize = MinMaxNormalizer(inplace=True)\n",
    "difference = FractionalDifference(difference_order=0.6,\n",
    "                                  inplace=True)\n",
    "feature_pipeline = FeaturePipeline(steps=[normalize, difference])\n",
    "\n",
    "reward_scheme = SimpleProfit()\n",
    "action_scheme = DiscreteActions(n_actions=20, instrument='ETH/BTC')\n",
    "\n",
    "exchange = FBMExchange(base_instrument='BTC',\n",
    "                       timeframe='1h',\n",
    "                       should_pretransform_obs=True)\n",
    "\n",
    "environment = TradingEnvironment(exchange=exchange,\n",
    "                                 action_scheme=action_scheme,\n",
    "                                 reward_scheme=reward_scheme,\n",
    "                                 feature_pipeline=feature_pipeline)\n",
    "\n",
    "agent_spec = {\n",
    "    \"type\": \"ppo_agent\",\n",
    "    \"step_optimizer\": {\n",
    "        \"type\": \"adam\",\n",
    "        \"learning_rate\": 1e-4\n",
    "    },\n",
    "    \"discount\": 0.99,\n",
    "    \"likelihood_ratio_clipping\": 0.2,\n",
    "}\n",
    "\n",
    "network_spec = [\n",
    "    dict(type='dense', size=64, activation=\"tanh\"),\n",
    "    dict(type='dense', size=32, activation=\"tanh\")\n",
    "]\n",
    "\n",
    "strategy = TensorforceTradingStrategy(environment=environment, agent_spec=agent_spec, network_spec=network_spec)\n",
    "\n",
    "performance = strategy.run(episodes=1, testing=True)\n",
    "\n",
    "performance[-5:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Finished running strategy.\n",
      "Total episodes: 10 (16660 timesteps).\n",
      "Average reward: 1.1827063286237227.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>balance</th>\n",
       "      <th>net_worth</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1660</th>\n",
       "      <td>11226.614898</td>\n",
       "      <td>11227.750341</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1661</th>\n",
       "      <td>11226.686075</td>\n",
       "      <td>11227.725535</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1662</th>\n",
       "      <td>11226.721664</td>\n",
       "      <td>11227.658857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1663</th>\n",
       "      <td>11226.739458</td>\n",
       "      <td>11227.908940</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1664</th>\n",
       "      <td>11223.960840</td>\n",
       "      <td>11225.188709</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           balance     net_worth\n",
       "1660  11226.614898  11227.750341\n",
       "1661  11226.686075  11227.725535\n",
       "1662  11226.721664  11227.658857\n",
       "1663  11226.739458  11227.908940\n",
       "1664  11223.960840  11225.188709"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from tensortrade.environments import TradingEnvironment\n",
    "from tensortrade.exchanges.simulated import FBMExchange\n",
    "from tensortrade.features.scalers import MinMaxNormalizer\n",
    "from tensortrade.features.stationarity import FractionalDifference\n",
    "from tensortrade.features import FeaturePipeline\n",
    "from tensortrade.rewards import SimpleProfit\n",
    "from tensortrade.actions import DiscreteActions\n",
    "from tensortrade.strategies import StableBaselinesTradingStrategy\n",
    "\n",
    "normalize = MinMaxNormalizer(inplace=True)\n",
    "difference = FractionalDifference(difference_order=0.6,\n",
    "                                  inplace=True)\n",
    "feature_pipeline = FeaturePipeline(steps=[normalize, difference])\n",
    "\n",
    "reward_scheme = SimpleProfit()\n",
    "action_scheme = DiscreteActions(n_actions=20, instrument='ETH/BTC')\n",
    "\n",
    "exchange = FBMExchange(base_instrument='BTC',\n",
    "                       timeframe='1h',\n",
    "                       should_pretransform_obs=True)\n",
    "\n",
    "environment = TradingEnvironment(exchange=exchange,\n",
    "                                 action_scheme=action_scheme,\n",
    "                                 reward_scheme=reward_scheme,\n",
    "                                 feature_pipeline=feature_pipeline)\n",
    "\n",
    "strategy = StableBaselinesTradingStrategy(environment=environment)\n",
    "\n",
    "performance = strategy.run(episodes=10)\n",
    "\n",
    "performance[-5:]"
   ]
  }
 ],
 "metadata": {
  "file_extension": ".py",
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  },
  "mimetype": "text/x-python",
  "name": "python",
  "npconvert_exporter": "python",
  "pygments_lexer": "ipython3",
  "version": 3
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
